opencode/NSCT---Neutral-Search-Crawler-Tool
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix search runs with empty results

Browse Source
This commit is contained in:
faligam
2026-09-07 12:46:06 +02:00
parent f921e978a0
commit e483c4d13f
6 changed files with 111 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
HANDOFF.md
View File

@@ -7,6 +7,30 @@
## Aktueller Stand — 2026-09-07
### Browser-E2E: Leeren Abschluss auf Suchausfall zurückgeführt und repariert — 2026-09-07
Der abgeschlossene Browser-Run `33328b0b-18df-48df-9f2d-8af077987ccc` hatte
einen validen Plan mit sechs Suchanfragen, aber keine Treffer: Quellen,
Claims, Evidenz und Bericht waren deshalb leer. SearXNG war erreichbar, seine
Default-Engine-Mischung lieferte im aktuellen Netz jedoch wegen CAPTCHA, 429
und Timeouts keine nutzbaren Ergebnisse. Der Orchestrator behandelte diesen
Fall fehlerhaft als erfolgreichen Run und erzeugte einen leeren Fallback-
Bericht.
- Ein providerweiter Trefferstand von null beendet den Run nun mit einem
eindeutigen Suchfehler, statt `completed` mit leerem Bericht zu speichern.
- Die SearXNG-Engine-Liste ist über `NSCT_SEARXNG_ENGINES` konfigurierbar.
Der lokale Compose-Default ist `bing,yahoo`; diese beiden Engines lieferten
im API-Container erfolgreich Ergebnisse. Betreiber können die Auswahl in
ihrer Umgebung überschreiben.
Validierung: 70 gezielte Tests (`test_backend_pipeline_repairs.py`,
`test_crawler.py`, `test_search.py`), Compose-Konfigurationsprüfung,
Docker-Rebuild und `/health` HTTP 200. Der direkte Provider-Probe aus dem
API-Container lieferte fünf Resultate. Für den nächsten Browser-E2E-Test eine
neue Recherche starten; ein API-Rebuild verwirft weiterhin alte In-Memory-
Run-IDs.
### Planungsinformationen in Research-Details umgesetzt — 2026-09-07
Issue [Frontend #1](https://git.frerkc.de/opencode/NSCT-FrontEnd/issues/1)

3
docker-compose.yml
View File

@@ -81,6 +81,9 @@ services:
NSCT_DB_URL: postgresql+asyncpg://${POSTGRES_USER:-nsct}:${POSTGRES_PASSWORD:-nsct_secret}@postgres:5432/${POSTGRES_DB:-nsct}
NSCT_LLM_MAX_CONCURRENCY: "${NSCT_LLM_MAX_CONCURRENCY:-3}"
NSCT_SEARXNG_BASE_URL: http://searxng:8080/
# The default SearXNG engine set is often rate-limited. Operators can
# override this comma-separated list in their environment.
NSCT_SEARXNG_ENGINES: "${NSCT_SEARXNG_ENGINES:-bing,yahoo}"
depends_on:
postgres:
condition: service_healthy

10
src/nsct/config.py
View File

@@ -62,6 +62,10 @@ class AppSettings(BaseModel):
postgres: DatabaseConfig = Field(default_factory=DatabaseConfig)
debug: bool = Field(default=False)
searxng_base_url: str | None = Field(default=None, description="SearXNG instance URL.")
searxng_engines: tuple[str, ...] = Field(
default=(),
description="Optional allow-list of SearXNG engines for web searches.",
)
@staticmethod
def from_env() -> "AppSettings":
@@ -98,6 +102,11 @@ class AppSettings(BaseModel):
debug = os.environ.get("NSCT_DEBUG", "false").lower() == "true"
searxng_url = os.environ.get("NSCT_SEARXNG_BASE_URL", None)
searxng_engines = tuple(
engine.strip()
for engine in os.environ.get("NSCT_SEARXNG_ENGINES", "").split(",")
if engine.strip()
)
return AppSettings(
llm=llm_cfg,
@@ -106,6 +115,7 @@ class AppSettings(BaseModel):
postgres=postgres_cfg,
debug=debug,
searxng_base_url=searxng_url,
searxng_engines=searxng_engines,
)
@property

21
src/nsct/orchestration/orchestrator.py
View File

@@ -307,7 +307,10 @@ class ResearchOrchestrator:
try:
from nsct.providers.searxng import SearXNGProvider
if self._config.searxng_base_url:
provider = SearXNGProvider(base_url=self._config.searxng_base_url)
provider = SearXNGProvider(
base_url=self._config.searxng_base_url,
engines=self._config.searxng_engines,
)
providers.append(provider)
except ImportError as exc:
logger.error("SearXNG provider is unavailable: %s", exc)
@@ -641,6 +644,22 @@ class ResearchOrchestrator:
}
self._search_results.append(result_dict)
if not self._search_results:
# An empty result set from every provider is not a completed
# research result. Continuing would make fetching,
# extraction, and synthesis silently produce an empty report
# and incorrectly mark the run as completed. A genuine
# "no evidence found" report needs an explicit, auditable
# outcome; provider-wide zero results are an operational
# failure until that outcome exists.
logger.error("Search returned no usable URLs from configured providers")
return {
"success": False,
"error": "Search returned no usable URLs from configured providers",
"data": {},
"url_count": 0,
}
logger.info("Search complete: %d unique URLs collected", len(self._search_results))
metrics.increment(C_SEARCH_QUERIES_TOTAL, len(plan_queries))
return {"success": True, "data": {"urls": self._search_results}, "url_count": len(self._search_results)}

15
src/nsct/providers/searxng.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
from collections.abc import Sequence
from typing import Any
import httpx
@@ -14,11 +15,17 @@ class SearXNGProvider(SearchProvider):
_provider_name = "searxng"
def __init__(self, base_url: str, timeout_seconds: float = 15.0) -> None:
def __init__(
self,
base_url: str,
timeout_seconds: float = 15.0,
engines: Sequence[str] | None = None,
) -> None:
if not base_url.strip():
raise ValueError("SearXNG base URL must not be empty")
self._base_url = base_url.rstrip("/")
self._timeout_seconds = timeout_seconds
self._engines = tuple(engine.strip() for engine in (engines or ()) if engine.strip())
self._client: httpx.AsyncClient | None = None
@property
@@ -30,9 +37,13 @@ class SearXNGProvider(SearchProvider):
async def search(
self, query: str, language: str = "de", max_results: int = 10
) -> list[NormalizedResult]:
params: dict[str, str] = {"q": query, "format": "json", "language": language}
if self._engines:
params["engines"] = ",".join(self._engines)
response = await self._http_client.get(
f"{self._base_url}/search",
params={"q": query, "format": "json", "language": language},
params=params,
# The bundled SearXNG instance enables bot detection and rejects
# requests without a client IP header. This is the loopback IP of
# the API-to-SearXNG hop, not an end-user supplied value.

40
tests/test_backend_pipeline_repairs.py
View File

@@ -69,6 +69,27 @@ def test_searxng_provider_normalizes_json_results() -> None:
asyncio.run(run())
def test_searxng_provider_sends_configured_engine_allow_list() -> None:
async def run() -> None:
seen_request: httpx.Request | None = None
def handler(request: httpx.Request) -> httpx.Response:
nonlocal seen_request
seen_request = request
return httpx.Response(200, json={"results": []})
provider = SearXNGProvider("http://searxng:8080", engines=("bing", "yahoo"))
provider._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
try:
await provider.search("test")
assert seen_request is not None
assert seen_request.url.params["engines"] == "bing,yahoo"
finally:
await provider._client.aclose()
asyncio.run(run())
def test_orchestrator_registers_searxng_provider() -> None:
orchestrator = ResearchOrchestrator(_config(), uuid4(), "test")
provider = orchestrator._get_multi_search().get_provider("searxng")
@@ -104,6 +125,25 @@ def test_searching_accepts_normalized_result_with_empty_snippet() -> None:
asyncio.run(run())
def test_searching_with_no_usable_results_fails_instead_of_completing_empty() -> None:
"""Provider-wide empty results must not become an empty completed report."""
async def run() -> None:
orchestrator = ResearchOrchestrator(_config(), uuid4(), "test")
multi_search = Mock()
multi_search.enabled_providers.return_value = ["searxng"]
multi_search.search = AsyncMock(return_value=[])
orchestrator._multi_search = multi_search
response = await orchestrator._step_searching()
assert response["success"] is False
assert response["url_count"] == 0
assert "no usable URLs" in response["error"]
assert orchestrator._search_results == []
asyncio.run(run())
def test_fetching_uses_actual_bytes_and_respects_remaining_source_budget() -> None:
async def run() -> None:
budget = HardBudgetConfig(max_sources=2, max_total_download_bytes=1_000)