Forked from ceveyne/user-docs
Forked from ceveyne/user-docs
dist / python / docling_parser.py
dist / python / docling_parser.py
#!/usr/bin/env python3
"""
Docling-based Document Parser (PDF and DOCX)
Uses IBM's docling library for professional document parsing with layout understanding.
DOCX needs no PDF-specific format_options below - DocumentConverter already dispatches it
to its own default WordFormatOption/SimplePipeline.
"""
import contextlib
import json
import os
import sys
from pathlib import Path
from typing import Dict, Any
STATUS_PREFIX = "[user-docs-status] "
def report_status(message: str) -> None:
print(f"{STATUS_PREFIX}{message}", file=sys.stderr, flush=True)
@contextlib.contextmanager
def _stdout_diverted_to_stderr():
"""
Guards the JSON-only stdout contract against library-internal writes
during conversion - docling_ibm_models' MatchingPostProcessor logger
defaults its own diagnostic handler to stream=sys.stdout instead of
stderr, which corrupts the final JSON print. Redirects the raw OS file
descriptor, not just sys.stdout, so it also catches native-extension
writes.
"""
sys.stdout.flush()
real_stdout_fd = os.dup(1)
try:
os.dup2(2, 1)
yield
finally:
sys.stdout.flush()
os.dup2(real_stdout_fd, 1)
os.close(real_stdout_fd)
try:
from docling.document_converter import DocumentConverter, PdfFormatOption
from docling.datamodel.base_models import InputFormat
from docling.datamodel.pipeline_options import PdfPipelineOptions
from docling_core.types.doc import ImageRefMode
except ImportError:
print(json.dumps({
"success": False,
"error": "docling not installed. Run: pip install docling",
"content": "",
"metadata": {}
}))
sys.exit(1)
def parse_document(file_path: str, extract_tables: bool = True, image_output_dir: str = None) -> Dict[str, Any]:
"""
Parse a document using docling
Args:
file_path: Path to the document file
extract_tables: Whether to extract and format tables (currently always enabled)
image_output_dir: When given, embedded pictures are extracted as PNG files into
this directory and the returned markdown references them
() instead of a
"<!-- image -->" placeholder. When omitted, behaviour is unchanged
(no picture extraction, placeholder-only markdown).
Returns:
Dictionary with parsed content and metadata
"""
try:
report_status("Preparing PDF extraction...")
pipeline_options = PdfPipelineOptions()
# The installed rapidocr build's torch backend does not support the
# PP-OCRv6/"small" model combination docling requests by default,
# which crashes conversion for every PDF (scanned or not). Our source
# PDFs are always real, text-layer documents, so OCR is not needed;
# disabling it avoids the crash and lets docling actually run.
pipeline_options.do_ocr = False
if image_output_dir:
# Only decode/keep picture bitmaps when a caller actually wants them.
pipeline_options.generate_picture_images = True
pipeline_options.images_scale = 2.0
report_status("Loading document analysis model...")
converter = DocumentConverter(
format_options={
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
}
)
# Convert document
report_status("Extracting PDF content and embedded images...")
result = converter.convert(file_path)
# Get the document
doc = result.document
if image_output_dir:
report_status("Writing extracted image files...")
out_dir = Path(image_output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
md_path = out_dir / "content.md"
# REFERENCED writes each picture as a PNG under out_dir/<md_stem>_artifacts/
# and rewrites the markdown to point at those files instead of a placeholder.
doc.save_as_markdown(md_path, image_mode=ImageRefMode.REFERENCED)
markdown_content = md_path.read_text(encoding="utf-8")
else:
markdown_content = doc.export_to_markdown()
# Get metadata
metadata = {
"format": Path(file_path).suffix.lstrip(".").lower(),
"strategy": "docling",
"title": getattr(doc, 'title', None) or Path(file_path).stem,
"page_count": result.input.page_count if hasattr(result, 'input') and hasattr(result.input, 'page_count') else 1,
"has_tables": '|' in markdown_content, # Simple heuristic: markdown tables have |
"char_count": len(markdown_content),
"image_count": len(doc.pictures),
}
return {
"success": True,
"content": markdown_content,
"metadata": metadata,
"error": None
}
except Exception as e:
return {
"success": False,
"error": str(e),
"content": "",
"metadata": {}
}
def main():
"""CLI interface for docling parser"""
if len(sys.argv) < 2:
print(json.dumps({
"success": False,
"error": "Usage: docling_parser.py <file_path> [extract_tables] [image_output_dir]",
"content": "",
"metadata": {}
}))
sys.exit(1)
file_path = sys.argv[1]
extract_tables = sys.argv[2].lower() == 'true' if len(sys.argv) > 2 else True
image_output_dir = sys.argv[3] if len(sys.argv) > 3 and sys.argv[3] else None
# Validate file exists
if not Path(file_path).exists():
print(json.dumps({
"success": False,
"error": f"File not found: {file_path}",
"content": "",
"metadata": {}
}))
sys.exit(1)
# Parse and output JSON
with _stdout_diverted_to_stderr():
result = parse_document(file_path, extract_tables, image_output_dir)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
#!/usr/bin/env python3
"""
Docling-based Document Parser (PDF and DOCX)
Uses IBM's docling library for professional document parsing with layout understanding.
DOCX needs no PDF-specific format_options below - DocumentConverter already dispatches it
to its own default WordFormatOption/SimplePipeline.
"""
import contextlib
import json
import os
import sys
from pathlib import Path
from typing import Dict, Any
STATUS_PREFIX = "[user-docs-status] "
def report_status(message: str) -> None:
print(f"{STATUS_PREFIX}{message}", file=sys.stderr, flush=True)
@contextlib.contextmanager
def _stdout_diverted_to_stderr():
"""
Guards the JSON-only stdout contract against library-internal writes
during conversion - docling_ibm_models' MatchingPostProcessor logger
defaults its own diagnostic handler to stream=sys.stdout instead of
stderr, which corrupts the final JSON print. Redirects the raw OS file
descriptor, not just sys.stdout, so it also catches native-extension
writes.
"""
sys.stdout.flush()
real_stdout_fd = os.dup(1)
try:
os.dup2(2, 1)
yield
finally:
sys.stdout.flush()
os.dup2(real_stdout_fd, 1)
os.close(real_stdout_fd)
try:
from docling.document_converter import DocumentConverter, PdfFormatOption
from docling.datamodel.base_models import InputFormat
from docling.datamodel.pipeline_options import PdfPipelineOptions
from docling_core.types.doc import ImageRefMode
except ImportError:
print(json.dumps({
"success": False,
"error": "docling not installed. Run: pip install docling",
"content": "",
"metadata": {}
}))
sys.exit(1)
def parse_document(file_path: str, extract_tables: bool = True, image_output_dir: str = None) -> Dict[str, Any]:
"""
Parse a document using docling
Args:
file_path: Path to the document file
extract_tables: Whether to extract and format tables (currently always enabled)
image_output_dir: When given, embedded pictures are extracted as PNG files into
this directory and the returned markdown references them
() instead of a
"<!-- image -->" placeholder. When omitted, behaviour is unchanged
(no picture extraction, placeholder-only markdown).
Returns:
Dictionary with parsed content and metadata
"""
try:
report_status("Preparing PDF extraction...")
pipeline_options = PdfPipelineOptions()
# The installed rapidocr build's torch backend does not support the
# PP-OCRv6/"small" model combination docling requests by default,
# which crashes conversion for every PDF (scanned or not). Our source
# PDFs are always real, text-layer documents, so OCR is not needed;
# disabling it avoids the crash and lets docling actually run.
pipeline_options.do_ocr = False
if image_output_dir:
# Only decode/keep picture bitmaps when a caller actually wants them.
pipeline_options.generate_picture_images = True
pipeline_options.images_scale = 2.0
report_status("Loading document analysis model...")
converter = DocumentConverter(
format_options={
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
}
)
# Convert document
report_status("Extracting PDF content and embedded images...")
result = converter.convert(file_path)
# Get the document
doc = result.document
if image_output_dir:
report_status("Writing extracted image files...")
out_dir = Path(image_output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
md_path = out_dir / "content.md"
# REFERENCED writes each picture as a PNG under out_dir/<md_stem>_artifacts/
# and rewrites the markdown to point at those files instead of a placeholder.
doc.save_as_markdown(md_path, image_mode=ImageRefMode.REFERENCED)
markdown_content = md_path.read_text(encoding="utf-8")
else:
markdown_content = doc.export_to_markdown()
# Get metadata
metadata = {
"format": Path(file_path).suffix.lstrip(".").lower(),
"strategy": "docling",
"title": getattr(doc, 'title', None) or Path(file_path).stem,
"page_count": result.input.page_count if hasattr(result, 'input') and hasattr(result.input, 'page_count') else 1,
"has_tables": '|' in markdown_content, # Simple heuristic: markdown tables have |
"char_count": len(markdown_content),
"image_count": len(doc.pictures),
}
return {
"success": True,
"content": markdown_content,
"metadata": metadata,
"error": None
}
except Exception as e:
return {
"success": False,
"error": str(e),
"content": "",
"metadata": {}
}
def main():
"""CLI interface for docling parser"""
if len(sys.argv) < 2:
print(json.dumps({
"success": False,
"error": "Usage: docling_parser.py <file_path> [extract_tables] [image_output_dir]",
"content": "",
"metadata": {}
}))
sys.exit(1)
file_path = sys.argv[1]
extract_tables = sys.argv[2].lower() == 'true' if len(sys.argv) > 2 else True
image_output_dir = sys.argv[3] if len(sys.argv) > 3 and sys.argv[3] else None
# Validate file exists
if not Path(file_path).exists():
print(json.dumps({
"success": False,
"error": f"File not found: {file_path}",
"content": "",
"metadata": {}
}))
sys.exit(1)
# Parse and output JSON
with _stdout_diverted_to_stderr():
result = parse_document(file_path, extract_tables, image_output_dir)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()