Fix research pipeline runtime failures

src/nsct/crawler/fetcher.py

@@ -98,6 +98,7 @@ class AsyncFetcher:
                 connect=float(os.environ.get("NSCT_CRAWLER_CONNECT_TIMEOUT", "10")),
                 read=float(os.environ.get("NSCT_CRAWLER_READ_TIMEOUT", "30")),
                 write=float(os.environ.get("NSCT_CRAWLER_WRITE_TIMEOUT", "10")),
+                pool=float(os.environ.get("NSCT_CRAWLER_POOL_TIMEOUT", "10")),
             )
             self._client = httpx.AsyncClient(
                 timeout=timeout_config,
@@ -304,4 +305,4 @@ class AsyncFetcher:
     @staticmethod
     def compute_content_hash(content: bytes) -> str:
         """Compute SHA256 hash of content."""
-        return hashlib.sha256(content).hexdigest()
+        return hashlib.sha256(content).hexdigest()

src/nsct/crawler/manager.py

@@ -36,7 +36,7 @@ class CrawlerManager:
     # Public API
     # ------------------------------------------------------------------
 
-    async def fetch_and_extract(self, url: str) -> NormalizedDocument:
+    async def fetch_and_extract(self, url: str, max_size: int = 0) -> NormalizedDocument:
         """Fetch a single URL and extract main content.
 
         Pipeline:
@@ -73,7 +73,7 @@ class CrawlerManager:
             logger.warning("DNS check warning for %s: %s", url, exc)
 
         # 3. HTTP fetch
-        result = await self.fetcher.fetch(url)
+        result = await self.fetcher.fetch(url, max_size=max_size)
 
         if result.status in (FetchStatus.BLOCKED, FetchStatus.ERROR, FetchStatus.TIMEOUT,
                              FetchStatus.SIZE_LIMIT_EXCEEDED, FetchStatus.CONTENT_TYPE_BLOCKED):
@@ -86,7 +86,7 @@ class CrawlerManager:
         # 4. Content-based extraction
         text = ""
         extraction_tool = ""
-        metadata: dict[str, Any] = {}
+        metadata: dict[str, Any] = {"download_bytes": len(content)}
         links: list[str] = []
         title = ""
 
@@ -131,7 +131,7 @@ class CrawlerManager:
 
         return doc
 
-    async def fetch_and_extract_many(self, urls: list[str]) -> list[NormalizedDocument]:
+    async def fetch_and_extract_many(self, urls: list[str], max_size: int = 0) -> list[NormalizedDocument]:
         """Fetch and extract multiple URLs in parallel.
 
         Each URL is processed independently — failures do not block others.
@@ -143,7 +143,7 @@ class CrawlerManager:
             List of NormalizedDocument, one per input URL (in order).
             Failed URLs produce error documents with error info in metadata.
         """
-        tasks = [self.fetch_and_extract(url) for url in urls]
+        tasks = [self.fetch_and_extract(url, max_size=max_size) for url in urls]
         results = await asyncio.gather(*tasks, return_exceptions=True)
 
         final_docs: list[NormalizedDocument] = []
@@ -262,4 +262,4 @@ class CrawlerManager:
         if lang_match:
             meta["language"] = lang_match.group(1)
 
-        return meta
+        return meta