opencode/NSCT---Neutral-Search-Crawler-Tool
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stage19: performance optimierung — LLM concurrency semaphore, priority (HIGH/NORMAL/LOW), limiter

Browse Source
This commit is contained in:
NSCT Agent
2026-08-28 17:28:31 +00:00
parent ce9b031ea0
commit 4335a40d68
7 changed files with 553 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/nsct/api/rest_research.py
View File

@@ -14,6 +14,7 @@ from pydantic import BaseModel, Field
from nsct.logging_config import set_request_ctx, clear_request_ctx, set_research_run_id, get_logger
from nsct.orchestration.budget import HardBudgetConfig
from nsct.orchestration.state import ResearchRunState
from nsct.providers.priority_queue import Priority
from nsct.metrics import (
metrics,
C_SEARCH_QUERIES_TOTAL,
@@ -410,6 +411,7 @@ async def _run_pipeline(run: _ResearchRunState, request: ResearchRequest, budget
query=request.query,
budget_config=budget,
depth=request.depth,
priority=Priority.NORMAL,
)
stage_ctx["stage"] = "planning"

5
src/nsct/orchestration/orchestrator.py
View File

@@ -33,6 +33,7 @@ from nsct.orchestration.budget import BudgetExhaustedError, BudgetTracker, HardB
from nsct.orchestration.models import ResearchRun
from nsct.orchestration.state import ResearchRunState, StateMachine
from nsct.providers.abstract import MultiProviderSearch, SearchProvider
from nsct.providers.priority_queue import Priority
try:
from nsct.stages.stage9_synthesis import SynthesisStage
@@ -61,6 +62,7 @@ class ResearchOrchestrator:
query: str,
budget_config: HardBudgetConfig | None = None,
depth: str = "normal",
priority: Priority = Priority.NORMAL,
) -> None:
"""Initialisiere den Orchestrator.
@@ -76,11 +78,14 @@ class ResearchOrchestrator:
Optionales Budget. Wird aus config abgeleitet, wenn None.
depth : str
Suchtiefe ("quick", "normal", "deep").
priority : Priority
LLM-Anfrage-Priorität für den Research-Pipeline.
"""
self._config = config
self._research_id = research_id
self._query = query
self._depth = depth
self._priority = priority
# Budget
if budget_config is not None:

3
src/nsct/providers/__init__.py
View File

@@ -15,6 +15,7 @@ from nsct.providers.llm import (
ProviderTimeoutError,
_LLMProviderImpl,
)
from nsct.providers.priority_queue import Priority, PriorityLimiter
from nsct.providers.metrics import ProviderMetrics
from nsct.providers.vision import _VisionProviderImpl
@@ -25,6 +26,8 @@ AudioProvider = _AudioProviderImpl
__all__ = [
"LLMProvider",
"_LLMProviderImpl",
"Priority",
"PriorityLimiter",
"ProviderError",
"ProviderHTTPError",
"ProviderModelNotFoundError",

85
src/nsct/providers/llm.py
View File

@@ -8,7 +8,7 @@ import logging
import os
import time
from abc import ABC, abstractmethod
from typing import Any, AsyncGenerator, cast
from typing import Any, AsyncGenerator
import httpx
from httpx import HTTPStatusError
@@ -18,6 +18,7 @@ from openai.types.chat import ChatCompletion, ChatCompletionChunk
from nsct.config import AppSettings
from nsct.providers.metrics import ProviderMetrics
from nsct.providers.priority_queue import Priority, PriorityLimiter
logger = logging.getLogger(__name__)
@@ -110,6 +111,7 @@ class _LLMProviderImpl(LLMProvider):
self._model: str = config.llm.model
self._max_concurrency: int = config.llm.max_concurrency
self._api_key: str = config.llm.get_secret()
self._limiter: PriorityLimiter | None = None
# Timeout config (seconds) — configurable via env, defaults here
self._connect_timeout: float = float(
@@ -121,6 +123,13 @@ class _LLMProviderImpl(LLMProvider):
self._client: AsyncOpenAI | None = None
@property
def limiter(self) -> PriorityLimiter:
"""Lazy-init the priority limiter."""
if self._limiter is None:
self._limiter = PriorityLimiter(self._max_concurrency)
return self._limiter
@property
def client(self) -> AsyncOpenAI:
"""Lazy-init the OpenAI client with connection pooling."""
@@ -205,8 +214,16 @@ class _LLMProviderImpl(LLMProvider):
temperature: float | None = None,
max_tokens: int | None = None,
response_format: dict[str, Any] | None = None,
priority: Priority = Priority.NORMAL,
) -> str:
"""Issue a chat-completions request and return the assistant text."""
"""Issue a chat-completions request and return the assistant text.
Parameters
----------
priority : Priority
Request priority. HIGH runs immediately, NORMAL waits for a slot,
LOW may be rejected when all slots are busy.
"""
model = model or self._model
async def _do() -> str:
@@ -239,7 +256,18 @@ class _LLMProviderImpl(LLMProvider):
choice = resp.choices[0]
return choice.message.content or ""
return await self._request_with_retry(_do)
limiter = self.limiter
try:
await limiter.acquire(priority)
except Exception as exc:
if priority == Priority.LOW:
logger.warning("LLM pool exhausted — LOW request rejected: %s", exc)
return ""
raise
try:
return await self._request_with_retry(_do)
finally:
limiter.release(priority)
async def stream_complete(
self,
@@ -247,28 +275,43 @@ class _LLMProviderImpl(LLMProvider):
model: str | None = None,
temperature: float | None = None,
max_tokens: int | None = None,
priority: Priority = Priority.NORMAL,
) -> AsyncGenerator[str, None]:
"""Stream chat completions. Yields delta content chunks."""
"""Stream chat completions. Yields delta content chunks.
Yields are deferred until the limiter releases so the slot is held
for the entire stream duration.
"""
model = model or self._model
tokens_input = 0
tokens_output = 0
start = time.monotonic()
limiter = self.limiter
chunks: list[str] = []
async for chunk in (await self._make_stream(messages, model, temperature, max_tokens)):
delta = chunk.choices[0].delta if chunk.choices and chunk.choices[0] else None
delta_text = delta.content if delta and delta.content else ""
if delta_text:
yield delta_text
tokens_output += 1
try:
await limiter.acquire(priority)
tokens_input = 0
tokens_output = 0
start = time.monotonic()
latency = time.monotonic() - start
for msg in messages:
tokens_input += len(msg.get("content", "").split())
await self._metrics.record_llm_request(
input_tokens=tokens_input,
output_tokens=tokens_output,
latency=latency,
)
async for chunk in (await self._make_stream(messages, model, temperature, max_tokens)):
delta = chunk.choices[0].delta if chunk.choices and chunk.choices[0] else None
delta_text = delta.content if delta and delta.content else ""
if delta_text:
chunks.append(delta_text)
tokens_output += 1
latency = time.monotonic() - start
for msg in messages:
tokens_input += len(msg.get("content", "").split())
await self._metrics.record_llm_request(
input_tokens=tokens_input,
output_tokens=tokens_output,
latency=latency,
)
finally:
self.limiter.release(priority)
for chunk_text in chunks:
yield chunk_text
async def _make_stream(
self,

123
src/nsct/providers/priority_queue.py Normal file
View File

@@ -0,0 +1,123 @@
"""Priority-aware limiter for LLM requests.
Priority levels:
- HIGH: bypasses the semaphore limit — always acquires immediately.
Uses a separate "priority" semaphore so HIGH requests do not
consume slots reserved for NORMAL.
- NORMAL: uses the main semaphore (slots are shared across all callers).
- LOW: rejected with LlmPriorityRejectError when the main semaphore is
fully occupied; waits otherwise.
"""
from __future__ import annotations
import asyncio
import enum
import logging
from typing import Any
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Priority enum
# ---------------------------------------------------------------------------
class Priority(int, enum.Enum):
"""Request priority for LLM calls."""
HIGH = 0
NORMAL = 1
LOW = 2
class LlmPriorityRejectError(Exception):
"""Raised when a LOW-priority request is rejected because all slots are busy."""
pass
# ---------------------------------------------------------------------------
# PriorityLimiter
# ---------------------------------------------------------------------------
class PriorityLimiter:
"""Semaphore-based rate-limiter with per-priority handling.
Parameters
----------
max_concurrency : int
Total slots for NORMAL + LOW combined. HIGH requests always succeed.
"""
def __init__(self, max_concurrency: int = 3) -> None:
self.max_concurrency = max_concurrency
# Main semaphore shared by NORMAL and LOW
self._semaphore: asyncio.Semaphore | None = None
self._lock = asyncio.Lock()
async def _ensure_sem(self) -> asyncio.Semaphore:
if self._semaphore is None:
async with self._lock:
if self._semaphore is None:
self._semaphore = asyncio.Semaphore(self.max_concurrency)
return self._semaphore
# -- public API --------------------------------------------------------
async def acquire(self, priority: Priority = Priority.NORMAL) -> None:
"""Acquire a slot for the given priority.
Raises
------
LlmPriorityRejectError
LOW request while all slots are occupied.
"""
if priority == Priority.HIGH:
# HIGH: always acquire immediately (no limit)
return
sem = await self._ensure_sem()
if priority == Priority.NORMAL:
await sem.acquire()
return
# LOW — reject if fully occupied
# Check semaphore value: if value is 0, no free slots
if sem._value is not None and sem._value <= 0:
raise LlmPriorityRejectError(
"All concurrency slots occupied — LOW request rejected"
)
# acquire_nowait() is the correct method name
await sem.acquire()
return
def release(self, priority: Priority = Priority.NORMAL) -> None:
"""Release a previously acquired slot."""
if priority == Priority.HIGH:
return
sem = self._semaphore
if sem is not None:
sem.release()
def active_count(self) -> int:
"""Number of currently acquired (NON-HIGH) slots."""
sem = self._semaphore
if sem is None:
return 0
return self.max_concurrency - sem._value # type: ignore[attr-defined]
def reset(self, max_concurrency: int | None = None) -> None:
"""Re-create internal semaphores with a new limit."""
limit = max_concurrency or self.max_concurrency
self.max_concurrency = limit
self._lock = asyncio.Lock()
self._semaphore = asyncio.Semaphore(limit)
def __aenter__(self) -> "PriorityLimiter":
return self
def __aexit__(self, *exc: Any) -> None:
pass

41
src/nsct/providers/semaphore.py Normal file
View File

@@ -0,0 +1,41 @@
"""LLM concurrency semaphore — hard limit on parallel LLM requests."""
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from typing import AsyncIterator
_semaphore: asyncio.Semaphore | None = None
_semaphore_lock: asyncio.Lock = asyncio.Lock()
_semaphore_max: int = 3
async def _get_semaphore(max_concurrency: int) -> asyncio.Semaphore:
"""Lazy-init global semaphore (thread-safe)."""
global _semaphore
async with _semaphore_lock:
if _semaphore is None:
_semaphore = asyncio.Semaphore(max_concurrency)
return _semaphore
@asynccontextmanager
async def llm_concurrency_limit(max_concurrency: int = 3) -> AsyncIterator[None]:
"""Context manager that limits concurrent LLM requests.
Usage::
async with llm_concurrency_limit(3):
await llm_provider.complete(...)
The semaphore is global and singleton — all callers share it.
"""
sem = await _get_semaphore(max_concurrency)
async with sem:
yield
def get_semaphore_max() -> int:
"""Return the current semaphore max (default 3)."""
return _semaphore_max