opencode/NSCT---Neutral-Search-Crawler-Tool
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add protected soft deletion for research runs

Browse Source
This commit is contained in:
faligam
2026-09-07 12:34:41 +02:00
parent 2c532bc9bb
commit f921e978a0
8 changed files with 445 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
API.md
View File

@@ -359,17 +359,34 @@ Identisch zu `GET /v1/research/{id}` — gleicher Response.
### DELETE /v1/research/{id}
**Beschreibung:** Löscht einen Research-Run (nur wenn nicht completed).
**Beschreibung:** Verschiebt einen Research-Run (auch einen abgeschlossenen) als
Soft Delete in den Papierkorb. Der Run ist anschließend in normalen Listen und
Abrufen nicht mehr sichtbar. Bei aktivem Löschschutz muss das Passwort im Body
übergeben werden; Administratoren benötigen es nicht.
**Response (200 OK):**
```json
{
"status": "deleted",
"status": "hidden",
"research_id": "uuid-..."
}
```
**Error (400):** Completed runs sind immutable.
### PUT /v1/research/{id}/deletion-protection
Setzt einen Löschschutz mit einem Passwort von mindestens acht Zeichen. Das
Passwort wird ausschließlich als scrypt-Hash gespeichert und nie ausgegeben.
Bei einer Änderung eines bestehenden Schutzes ist `current_password` nötig.
### GET /v1/admin/research/deleted
Administrator-Endpunkt für den Papierkorb. Die Administratorrolle wird lokal
mit `nsct-api-key grant-admin <username>` vergeben.
### DELETE /v1/admin/research/{id}
Endgültiges Löschen eines bereits ausgeblendeten Eintrags. Sowohl Soft Delete
als auch Purge werden in `research_deletion_audit` protokolliert.
---

25
src/nsct/admin.py
View File

@@ -11,7 +11,7 @@ from sqlalchemy import select
from nsct.config import AppSettings
from nsct.security.api_keys import create_api_key, revoke_api_key
from nsct.storage.engine import close_engine, get_engine, get_session_factory
from nsct.storage.models import APIKeyModel, UserModel
from nsct.storage.models import APIKeyModel, AdminUserModel, UserModel
def _parse_expiry(value: str | None) -> datetime | None:
@@ -51,6 +51,25 @@ async def _run(args: argparse.Namespace) -> int:
await session.commit()
print(f"Revoked API key {args.key_id}.")
return 0
if args.command in {"grant-admin", "revoke-admin"}:
user = await session.scalar(select(UserModel).where(UserModel.username == args.username))
if user is None:
print("No user found for that username.")
return 1
grant = await session.get(AdminUserModel, user.id)
if args.command == "grant-admin":
if grant is None:
session.add(AdminUserModel(user_id=user.id))
await session.commit()
print(f"Granted administrator privileges to {user.username}.")
return 0
if grant is None:
print(f"{user.username} is not an administrator.")
return 1
await session.delete(grant)
await session.commit()
print(f"Revoked administrator privileges from {user.username}.")
return 0
rows = await session.execute(
select(APIKeyModel, UserModel)
@@ -77,6 +96,10 @@ def main() -> int:
create.add_argument("--expires-at", help="ISO-8601 timestamp; omitted means no expiry")
revoke = commands.add_parser("revoke", help="Revoke a key by its public key_id")
revoke.add_argument("key_id")
grant_admin = commands.add_parser("grant-admin", help="Grant administrator privileges to an existing user")
grant_admin.add_argument("username")
revoke_admin = commands.add_parser("revoke-admin", help="Revoke administrator privileges from a user")
revoke_admin.add_argument("username")
commands.add_parser("list", help="List key metadata (never plaintext or hashes)")
args = parser.parse_args()
try:

245
src/nsct/api/rest_research.py
View File

@@ -8,8 +8,10 @@ import uuid
from datetime import datetime, timezone
from typing import Any
from fastapi import BackgroundTasks, APIRouter, Depends, HTTPException
from fastapi import BackgroundTasks, APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from nsct.logging_config import set_request_ctx, clear_request_ctx, set_research_run_id, get_logger
from nsct.orchestration.budget import HardBudgetConfig
@@ -27,6 +29,14 @@ from nsct.metrics import (
G_ACTIVE_RESEARCH_RUNS,
)
from nsct.security.api_keys import require_api_key
from nsct.security.deletion_protection import (
hash_deletion_password,
is_administrator,
may_manage_retention,
verify_deletion_password,
)
from nsct.storage.engine import get_optional_session
from nsct.storage.models import APIKeyModel, ResearchDeletionAuditModel, ResearchRetentionModel
logger = logging.getLogger(__name__)
@@ -114,6 +124,7 @@ class ResearchListItem(BaseModel):
created_at: str
source_count: int = 0
claim_count: int = 0
is_deletion_protected: bool = False
class ResearchListResponse(BaseModel):
@@ -150,6 +161,27 @@ class StatusResponse(BaseModel):
error: str | None = None
is_completed: bool = False
is_running: bool = False
is_deletion_protected: bool = False
class DeletionProtectionRequest(BaseModel):
"""Set or replace a password that guards a research against deletion."""
password: str = Field(..., min_length=8, max_length=256)
current_password: str | None = Field(default=None, max_length=256)
class DeletionRequest(BaseModel):
"""Optional password used when deleting a protected research."""
password: str | None = Field(default=None, max_length=256)
class DeletedResearchItem(BaseModel):
research_id: str
query: str
hidden_at: str
is_deletion_protected: bool
class SourceListItem(BaseModel):
@@ -272,6 +304,8 @@ class _ResearchRunState(BaseModel):
evidence_scores: list[dict[str, Any]] = Field(default_factory=list)
report: dict[str, Any] | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
owner_user_id: str | None = None
is_hidden: bool = False
# Global in-memory store.
@@ -283,7 +317,7 @@ def _save_run(run: _ResearchRunState) -> None:
def _load_run(research_id: str) -> _ResearchRunState:
if research_id not in _research_store:
if research_id not in _research_store or _research_store[research_id].is_hidden:
raise HTTPException(status_code=404, detail=f"Research run {research_id} not found")
return _research_store[research_id]
@@ -292,6 +326,32 @@ def _now() -> str:
return datetime.now(timezone.utc).isoformat()
async def _retention_record(session: AsyncSession | None, research_id: str) -> ResearchRetentionModel | None:
if session is None:
return None
return await session.scalar(select(ResearchRetentionModel).where(ResearchRetentionModel.research_id == research_id))
async def _is_deletion_protected(session: AsyncSession | None, research_id: str) -> bool:
record = await _retention_record(session, research_id)
return bool(record and record.deletion_password_hash)
async def _require_retention_access(
session: AsyncSession | None, research_id: str, api_key: APIKeyModel | None
) -> tuple[ResearchRetentionModel | None, bool]:
"""Return retention metadata and admin status, rejecting another user's run."""
record = await _retention_record(session, research_id)
# Older in-memory-only runs have no owner metadata. Keep their legacy
# behavior while all newly authenticated runs receive a persistent record.
if record is None:
return None, False
admin = await is_administrator(session, api_key.user_id if api_key else None) if session else False
if not may_manage_retention(record, api_key.user_id if api_key else None, admin):
raise HTTPException(status_code=403, detail="You may only manage your own research runs.")
return record, admin
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@@ -302,9 +362,23 @@ def _now() -> str:
response_model=ResearchListResponse,
summary="List all research runs (paginated)",
)
async def list_research(limit: int = 10, offset: int = 0) -> ResearchListResponse:
async def list_research(
limit: int = 10,
offset: int = 0,
session: AsyncSession | None = Depends(get_optional_session),
api_key: APIKeyModel | None = Depends(require_api_key),
) -> ResearchListResponse:
"""Liste aller Research-Runs (paginiert)."""
all_runs = list(_research_store.values())
all_runs = [r for r in _research_store.values() if not r.is_hidden]
protected_ids: set[str] = set()
if all_runs and session is not None:
rows = await session.scalars(
select(ResearchRetentionModel.research_id).where(
ResearchRetentionModel.research_id.in_([r.research_id for r in all_runs]),
ResearchRetentionModel.deletion_password_hash.is_not(None),
)
)
protected_ids = set(rows)
total = len(all_runs)
all_runs.sort(key=lambda r: r.created_at, reverse=True)
sliced = all_runs[offset : offset + limit]
@@ -317,6 +391,7 @@ async def list_research(limit: int = 10, offset: int = 0) -> ResearchListRespons
created_at=r.created_at,
source_count=len(r.sources),
claim_count=len(r.claims),
is_deletion_protected=r.research_id in protected_ids,
)
for r in sliced
]
@@ -331,6 +406,8 @@ async def list_research(limit: int = 10, offset: int = 0) -> ResearchListRespons
async def start_research(
request: ResearchRequest,
bg: BackgroundTasks,
session: AsyncSession | None = Depends(get_optional_session),
api_key: APIKeyModel | None = Depends(require_api_key),
) -> ResearchCreateResponse:
"""Startet eine neue Recherche asynchron.
@@ -376,8 +453,20 @@ async def start_research(
state=ResearchRunState.CREATED.value,
created_at=created_at,
updated_at=created_at,
owner_user_id=api_key.user_id if api_key else None,
)
_save_run(run)
# The retention row is intentionally created before background processing:
# an immediately deleted run still has an accountable lifecycle record.
if api_key is not None and session is not None:
session.add(
ResearchRetentionModel(
research_id=research_id,
owner_user_id=api_key.user_id,
query=request.query,
)
)
await session.flush()
# Increment active gauge
import nsct.metrics as _m
@@ -528,7 +617,10 @@ async def _run_pipeline(run: _ResearchRunState, request: ResearchRequest, budget
response_model=StatusResponse,
summary="Get research run metadata",
)
async def get_research(research_id: str) -> StatusResponse:
async def get_research(
research_id: str,
session: AsyncSession | None = Depends(get_optional_session),
) -> StatusResponse:
"""Metadaten eines Research-Runs abrufen."""
run = _load_run(research_id)
return StatusResponse(
@@ -548,6 +640,7 @@ async def get_research(research_id: str) -> StatusResponse:
ResearchRunState.FAILED.value,
ResearchRunState.CANCELLED.value,
),
is_deletion_protected=await _is_deletion_protected(session, research_id),
)
@@ -556,7 +649,10 @@ async def get_research(research_id: str) -> StatusResponse:
response_model=StatusResponse,
summary="Get research status",
)
async def get_status(research_id: str) -> StatusResponse:
async def get_status(
research_id: str,
session: AsyncSession | None = Depends(get_optional_session),
) -> StatusResponse:
"""Status eines Research-Runs abrufen."""
run = _load_run(research_id)
return StatusResponse(
@@ -576,6 +672,7 @@ async def get_status(research_id: str) -> StatusResponse:
ResearchRunState.FAILED.value,
ResearchRunState.CANCELLED.value,
),
is_deletion_protected=await _is_deletion_protected(session, research_id),
)
@@ -700,20 +797,130 @@ async def get_report(research_id: str) -> ReportResponse:
@router.delete(
"/v1/research/{research_id}",
response_model=dict[str, str],
summary="Cancel and delete a research run",
summary="Soft-delete a research run",
)
async def delete_research(research_id: str) -> dict[str, str]:
"""Research-Run löschen (nur bei 'created' oder 'failed').
Läuft ein Run gerade, wird er zunächst abgebrochen.
"""
async def delete_research(
research_id: str,
request: DeletionRequest | None = None,
session: AsyncSession | None = Depends(get_optional_session),
api_key: APIKeyModel | None = Depends(require_api_key),
) -> dict[str, str]:
"""Hide a run without destroying it; protected runs require their password."""
run = _load_run(research_id)
if run.state == ResearchRunState.COMPLETED.value:
raise HTTPException(
status_code=400,
detail="Cannot delete a completed research run. It is immutable.",
)
record, admin = await _require_retention_access(session, research_id, api_key)
if record and record.deletion_password_hash and not admin:
if request is None or not await verify_deletion_password(request.password or "", record.deletion_password_hash):
raise HTTPException(status_code=403, detail="A valid deletion-protection password is required.")
run.is_hidden = True
run.state = ResearchRunState.CANCELLED.value
run.updated_at = _now()
del _research_store[research_id]
return {"status": "deleted", "research_id": research_id}
_save_run(run)
if record and session is not None:
record.is_hidden = True
record.hidden_at = datetime.now(timezone.utc).replace(tzinfo=None)
session.add(
ResearchDeletionAuditModel(
research_id=research_id,
actor_user_id=api_key.user_id if api_key else None,
action="soft_deleted",
)
)
await session.flush()
return {"status": "hidden", "research_id": research_id}
@router.put(
"/v1/research/{research_id}/deletion-protection",
response_model=dict[str, bool],
summary="Set or replace deletion protection",
)
async def set_deletion_protection(
research_id: str,
request: DeletionProtectionRequest,
session: AsyncSession | None = Depends(get_optional_session),
api_key: APIKeyModel | None = Depends(require_api_key),
) -> dict[str, bool]:
"""Set a password without ever returning it or its hash to the client."""
_load_run(research_id)
record, admin = await _require_retention_access(session, research_id, api_key)
if record is None:
raise HTTPException(status_code=409, detail="Research retention metadata is not available for this legacy run.")
if record.deletion_password_hash and not admin:
if not await verify_deletion_password(request.current_password or "", record.deletion_password_hash):
raise HTTPException(status_code=403, detail="The current deletion-protection password is required.")
record.deletion_password_hash = await hash_deletion_password(request.password)
await session.flush()
return {"is_deletion_protected": True}
@router.delete(
"/v1/research/{research_id}/deletion-protection",
response_model=dict[str, bool],
summary="Remove deletion protection",
)
async def remove_deletion_protection(
research_id: str,
request: DeletionRequest,
session: AsyncSession | None = Depends(get_optional_session),
api_key: APIKeyModel | None = Depends(require_api_key),
) -> dict[str, bool]:
_load_run(research_id)
record, admin = await _require_retention_access(session, research_id, api_key)
if record is None or not record.deletion_password_hash:
return {"is_deletion_protected": False}
if not admin and not await verify_deletion_password(request.password or "", record.deletion_password_hash):
raise HTTPException(
status_code=403,
detail="A valid deletion-protection password is required.",
)
record.deletion_password_hash = None
await session.flush()
return {"is_deletion_protected": False}
async def _require_admin(session: AsyncSession | None, api_key: APIKeyModel | None) -> None:
if session is None or not await is_administrator(session, api_key.user_id if api_key else None):
raise HTTPException(status_code=403, detail="Administrator privileges are required.")
@router.get("/v1/admin/research/deleted", response_model=list[DeletedResearchItem], summary="List soft-deleted research")
async def list_deleted_research(
session: AsyncSession | None = Depends(get_optional_session), api_key: APIKeyModel | None = Depends(require_api_key)
) -> list[DeletedResearchItem]:
await _require_admin(session, api_key)
assert session is not None
records = await session.scalars(
select(ResearchRetentionModel)
.where(ResearchRetentionModel.is_hidden.is_(True))
.order_by(ResearchRetentionModel.hidden_at.desc())
)
return [
DeletedResearchItem(
research_id=record.research_id,
query=record.query,
hidden_at=(record.hidden_at or record.updated_at).replace(tzinfo=timezone.utc).isoformat(),
is_deletion_protected=bool(record.deletion_password_hash),
)
for record in records
]
@router.delete("/v1/admin/research/{research_id}", response_model=dict[str, str], summary="Permanently purge hidden research")
async def purge_deleted_research(
research_id: str,
session: AsyncSession | None = Depends(get_optional_session),
api_key: APIKeyModel | None = Depends(require_api_key),
) -> dict[str, str]:
await _require_admin(session, api_key)
assert session is not None
record = await _retention_record(session, research_id)
if record is None or not record.is_hidden:
raise HTTPException(status_code=404, detail="No hidden research run found.")
session.add(
ResearchDeletionAuditModel(research_id=research_id, actor_user_id=api_key.user_id if api_key else None, action="purged")
)
await session.delete(record)
_research_store.pop(research_id, None)
await session.flush()
return {"status": "purged", "research_id": research_id}

63
src/nsct/security/deletion_protection.py Normal file
View File

@@ -0,0 +1,63 @@
"""Password hashing and authorization helpers for research deletion protection."""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.concurrency import run_in_threadpool
from nsct.storage.models import AdminUserModel, ResearchRetentionModel
_SCRYPT_N = 2**15
_SCRYPT_R = 8
_SCRYPT_P = 1
def _hash_password(password: str, salt: bytes | None = None) -> str:
salt = salt or secrets.token_bytes(16)
digest = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=_SCRYPT_N, r=_SCRYPT_R, p=_SCRYPT_P, maxmem=64 * 1024 * 1024
)
return "scrypt$32768$8$1${}${}".format(
base64.urlsafe_b64encode(salt).decode("ascii"),
base64.urlsafe_b64encode(digest).decode("ascii"),
)
def _verify_password(password: str, stored_hash: str) -> bool:
try:
algorithm, n, r, p, salt_b64, digest_b64 = stored_hash.split("$")
if algorithm != "scrypt":
return False
candidate = hashlib.scrypt(
password.encode("utf-8"),
salt=base64.urlsafe_b64decode(salt_b64.encode("ascii")),
n=int(n), r=int(r), p=int(p), maxmem=64 * 1024 * 1024,
)
return hmac.compare_digest(candidate, base64.urlsafe_b64decode(digest_b64.encode("ascii")))
except (ValueError, TypeError, UnicodeError):
return False
async def hash_deletion_password(password: str) -> str:
return await run_in_threadpool(_hash_password, password)
async def verify_deletion_password(password: str, stored_hash: str | None) -> bool:
return bool(stored_hash) and await run_in_threadpool(_verify_password, password, stored_hash)
async def is_administrator(session: AsyncSession, user_id: str | None) -> bool:
if not user_id:
return False
return await session.scalar(select(AdminUserModel.user_id).where(AdminUserModel.user_id == user_id)) is not None
def may_manage_retention(record: ResearchRetentionModel | None, user_id: str | None, is_admin: bool) -> bool:
"""Owners manage their own run; an explicit administrator may manage any run."""
return bool(record) and (is_admin or (user_id is not None and record.owner_user_id == user_id))

35
src/nsct/storage/engine.py
View File

@@ -9,7 +9,13 @@ from sqlalchemy import event
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from nsct.config import AppSettings
from nsct.storage.models import APIKeyModel, UserModel
from nsct.storage.models import (
APIKeyModel,
AdminUserModel,
ResearchDeletionAuditModel,
ResearchRetentionModel,
UserModel,
)
logger = logging.getLogger(__name__)
@@ -38,14 +44,19 @@ async def get_engine(config: AppSettings) -> None:
engine_options.update(pool_size=10, max_overflow=20, pool_recycle=1800)
_engine = create_async_engine(db_url, **engine_options)
# Auth is the only persistent HTTP concern currently using this engine.
# Other legacy model metadata contains tables that are not yet deployable
# as a single schema, so creating all of it here can prevent API startup.
# Only tables used by the HTTP control plane are created here. The legacy
# evidence models deliberately remain outside this small deployable schema.
async with _engine.begin() as conn:
await conn.run_sync(
lambda sync_conn: UserModel.metadata.create_all(
sync_conn,
tables=[UserModel.__table__, APIKeyModel.__table__],
tables=[
UserModel.__table__,
APIKeyModel.__table__,
AdminUserModel.__table__,
ResearchRetentionModel.__table__,
ResearchDeletionAuditModel.__table__,
],
)
)
@@ -77,6 +88,20 @@ async def get_session() -> AsyncGenerator[AsyncSession, None]:
await session.close()
async def get_optional_session() -> AsyncGenerator[AsyncSession | None, None]:
"""Yield a session when persistence is configured, otherwise ``None``.
This keeps the legacy in-memory research endpoints usable in isolated unit
tests while production requests always receive a database session through
API-key authentication.
"""
if _session_factory is None:
yield None
return
async for session in get_session():
yield session
def get_session_factory() -> async_sessionmaker[AsyncSession]:
"""Return the initialized factory for trusted administrative commands."""
if _session_factory is None:

56
src/nsct/storage/models.py
View File

@@ -126,6 +126,62 @@ class APIKeyModel(Base):
)
# ---------------------------------------------------------------------------
# Research retention and deletion audit
# ---------------------------------------------------------------------------
class ResearchRetentionModel(Base):
"""Persistent lifecycle metadata for a research run.
The pipeline payload is currently held by the runtime store, but retention
decisions must survive an API restart. Passwords are stored only as a
salted hash; ``is_hidden`` implements the user-facing soft delete.
"""
__tablename__ = "research_retention"
research_id = Column(String(36), primary_key=True)
owner_user_id = Column(String(36), ForeignKey("users.id"), nullable=True)
query = Column(Text, nullable=False)
deletion_password_hash = Column(String(255), nullable=True)
is_hidden = Column(Boolean, nullable=False, default=False)
hidden_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
__table_args__ = (
Index("ix_research_retention_owner", "owner_user_id"),
Index("ix_research_retention_hidden", "is_hidden"),
)
class ResearchDeletionAuditModel(Base):
"""Append-only audit trail for soft and final research deletions."""
__tablename__ = "research_deletion_audit"
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
research_id = Column(String(36), nullable=False)
actor_user_id = Column(String(36), ForeignKey("users.id"), nullable=True)
action = Column(String(32), nullable=False)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
__table_args__ = (
Index("ix_research_deletion_audit_research", "research_id"),
Index("ix_research_deletion_audit_created", "created_at"),
)
class AdminUserModel(Base):
"""Explicit administrator grant, separate from API-key ownership."""
__tablename__ = "admin_users"
user_id = Column(String(36), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True)
granted_at = Column(DateTime, nullable=False, default=datetime.utcnow)
# ---------------------------------------------------------------------------
# SearchQuery
# ---------------------------------------------------------------------------

19
tests/test_deletion_protection.py Normal file
View File

@@ -0,0 +1,19 @@
"""Focused tests for deletion-protection primitives without external providers."""
from __future__ import annotations
import pytest
from nsct.security.deletion_protection import hash_deletion_password, verify_deletion_password
@pytest.mark.asyncio
async def test_deletion_password_is_salted_and_verifiable() -> None:
password = "a-long-enough-deletion-password"
first = await hash_deletion_password(password)
second = await hash_deletion_password(password)
assert first != password
assert first != second
assert await verify_deletion_password(password, first)
assert not await verify_deletion_password("incorrect-password", first)

16
tests/test_rest_research.py
View File

@@ -413,8 +413,8 @@ def test_get_report_not_completed(client: TestClient) -> None:
# ---------------------------------------------------------------------------
def test_delete_research_active_fails(client: TestClient) -> None:
"""DELETE handles both active and finished research."""
def test_delete_research_soft_deletes_all_states(client: TestClient) -> None:
"""DELETE hides a run, including a completed one, instead of destroying it."""
from nsct.api.rest_research import _research_store, ResearchRunState
_research_store.clear()
@@ -434,14 +434,12 @@ def test_delete_research_active_fails(client: TestClient) -> None:
# Already deleted by previous tests — skip
return
# If completed, delete should return 400 (immutable)
# If failed/created, delete should succeed
resp = client.delete(f"/v1/research/{research_id}")
assert resp.status_code in (200, 400)
if resp.status_code == 200:
body = resp.json()
assert body["status"] == "deleted"
assert body["research_id"] == research_id
assert resp.status_code == 200
body = resp.json()
assert body["status"] == "hidden"
assert body["research_id"] == research_id
assert research_id not in _research_store or _research_store[research_id].is_hidden
def test_delete_research_not_found(client: TestClient) -> None: