a8595cc950
- AsyncFetcher: Sicheres HTTP-Fetching mit SSRF-Schutz, Connection Pooling (50/10), Timeout, Redirect Limit, Rate Limiting, User-Agent - Content-Extraction: trafilatura für HTML→Text, BeautifulSoup4 Fallback - PDF-Extraction: pdfminer.six mit Error-Handling - NormalizedDocument: Schema (url, title, text, metadata, links, content_hash, extraction_tool, extracted_at, word_count) - Crawler-Manager: SSRF-Check → robots.txt → HTTP-Fetch → Extraction → Normalization (Batch-fähig, Error-Isolation pro Fetch) - Security-Policy: SSRF-Schutz (RFC1918, Cloud Metadata, file://, ftp://, localhost), URL-Validation (nur http/https) - Crawler-Endpoints: POST /crawler/fetch, /crawler/fetch/batch, /crawler/validate-url - Test-Cases: SSRF-Schutz, Content Extraction, NormalizedDocument, Error Handling, Content Hash Determinismus
118 lines
3.2 KiB
Python
118 lines
3.2 KiB
Python
"""PDF content extraction for NSCT crawler.
|
|
|
|
Extracts text from PDF documents using pdfminer.six or pdfplumber
|
|
as fallback. Returns structured error markers when text extraction
|
|
is not possible.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import logging
|
|
from typing import BinaryIO
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
try:
|
|
from pdfminer.high_level import extract_text as pdfminer_extract_text
|
|
from pdfminer.layout import LAParams
|
|
|
|
_HAS_PDFMINER = True
|
|
except ImportError:
|
|
_HAS_PDFMINER = False
|
|
pdfminer_extract_text = None # type: ignore[assignment,misc]
|
|
|
|
try:
|
|
import pdfplumber
|
|
|
|
_HAS_PDFPLUMBER = True
|
|
except ImportError:
|
|
_HAS_PDFPLUMBER = False
|
|
pdfplumber = None # type: ignore[assignment,misc]
|
|
|
|
try:
|
|
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
|
|
from pdfminer.pdfpage import PDFPage
|
|
from pdfminer.converter import XMLConverter, TextConverter
|
|
from pdfminer.layout import LAParams
|
|
|
|
_HAS_PDFMINER_FULL = True
|
|
except ImportError:
|
|
_HAS_PDFMINER_FULL = False
|
|
PDFResourceManager = None # type: ignore[assignment,misc]
|
|
PDFPage = None # type: ignore[assignment,misc]
|
|
PDFPageInterpreter = None # type: ignore[assignment,misc]
|
|
XMLConverter = None # type: ignore[assignment,misc]
|
|
TextConverter = None # type: ignore[assignment,misc]
|
|
|
|
|
|
def extract_pdf_from_bytes(pdf_bytes: bytes) -> str:
|
|
"""Extract text content from raw PDF bytes.
|
|
|
|
Priority: pdfminer.six -> pdfplumber -> minimal fallback.
|
|
|
|
Args:
|
|
pdf_bytes: Raw PDF content.
|
|
|
|
Returns:
|
|
Extracted text content, or an error marker string if extraction fails.
|
|
"""
|
|
if not pdf_bytes:
|
|
return "pdf_extract_failed"
|
|
|
|
content_types = "pdf_no_text"
|
|
|
|
# --- Try pdfminer.six ---
|
|
if _HAS_PDFMINER:
|
|
try:
|
|
stream = io.BytesIO(pdf_bytes)
|
|
text = pdfminer_extract_text(
|
|
stream,
|
|
laparams=LAParams(
|
|
line_margin=0.5,
|
|
word_margin=0.1,
|
|
char_margin=2.0,
|
|
boxes_flow=0.5,
|
|
),
|
|
)
|
|
if text and text.strip():
|
|
return text.strip()
|
|
except Exception as exc:
|
|
logger.debug("pdfminer extraction failed: %s", exc)
|
|
|
|
# --- Try pdfplumber ---
|
|
if _HAS_PDFPLUMBER:
|
|
try:
|
|
stream = io.BytesIO(pdf_bytes)
|
|
with pdfplumber.open(stream) as pdf:
|
|
parts: list[str] = []
|
|
for page in pdf.pages:
|
|
page_text = page.extract_text()
|
|
if page_text:
|
|
parts.append(page_text)
|
|
if parts:
|
|
text = "\n\n".join(parts)
|
|
return text.strip()
|
|
except Exception as exc:
|
|
logger.debug("pdfplumber extraction failed: %s", exc)
|
|
|
|
return f"pdf_extract_failed"
|
|
|
|
|
|
def detect_pdf_content_type(pdf_bytes: bytes) -> str:
|
|
"""Detect the content type of PDF bytes.
|
|
|
|
Args:
|
|
pdf_bytes: Raw PDF content.
|
|
|
|
Returns:
|
|
Content type string.
|
|
"""
|
|
if not pdf_bytes:
|
|
return "unknown"
|
|
|
|
# Check PDF magic bytes
|
|
if pdf_bytes[:4] == b"%PDF":
|
|
return "application/pdf"
|
|
|
|
return "unknown" |